#' @author edvard
#' @time 2017.03.01

### Crawling the category of application ###
# This surprised me that wandoujia doesn't have any restricition
# The "com.xiachufang" is the package name of the application
base_url <- "http://www.wandoujia.com/apps/"
app.info <- data.frame(app_name = "", app_category = "",app_downloads = "", stringsAsFactors = F)
tmp <- application.package[,1] %>% as.character()
for(i in 1:(tmp %>% length())){
  r <- GET(paste(base_url, tmp[i], sep = "") %>% print())

  if(200 == status_code(r)){
    content <- content(r, "text", encoding = "UTF-8")
    web <- read_html(content, encoding = "UTF-8")
    categories <- web %>% html_nodes("dd.tag-box a") %>% html_text()  # Corresponding app category
    downloads <- web %>% html_node("div.num-list span.item i") %>% html_text()
    name <- web %>% html_node("p.app-name span.title") %>% html_text()
    
    # try({
    #   # If only very few people download, then remove it
    #   tmp.bool <- downloads %>% as.numeric() %>% is.numeric()
    #   if(tmp.bool)
    #     next
    #   
    # }, silent = T)
    
    app.info[i, 1] = name
    app.info[i, 2] = paste(categories, collapse = " ")
    app.info[i, 3] = downloads
  }else{
    print("failed...")
  }
  
  # Stop 3 seconds
  Sys.sleep(1.5)
}

app.info <- data.frame(app.info, app_package = application.package[,1])

cache <- app.info
# app.info <- cache
app.info <- app.info %>% filter(!is.na(app_name))
# 单位有亿万，在这里只处理万 
app.info <- app.info[-grep("亿", app.info$app_downloads),]
str.downloads <- strsplit(app.info$app_downloads, " ") %>% unlist()
for(i in 1:length(str.downloads)){
  app.info[i, 3] <- str.downloads[2*i - 1]
}
app.info <- app.info %>% filter(!is.na(app_name))

app.info$app_downloads <- app.info$app_downloads %>% as.numeric()
app.info <- mutate(
  app.info,
  inverse_weight = log(10000/app_downloads, 10)
)

cache <- app.info[,-c(3,4)]

write.csv(app.info, "cache/app_info.csv", row.names = F)


